<%@page import="org.datanucleus.query.evaluator.memory.StringLengthMethodEvaluator"%>
<%@ page import="agentFree2.*,java.net.*,java.io.*,java.nio.charset.*,java.nio.*,java.util.Date,java.util.logging.Logger,java.util.regex.Pattern,java.util.regex.Matcher,javax.jdo.PersistenceManager,javax.servlet.http.*" %>
<%
	//Stage1. Parsing Url Source and getting two phone numbers.

	//Pattern agentBlock = Pattern.compile("<p>(8-029) 222-22-55, (8-0162) 22-00-55, <a href=\"http://realt.by/agencies/\\?agency_id=114\" target=\"_blank\" >.{5,30</a>");
	//Pattern agentBlock = Pattern.compile("<p>.{5,50}<a href=\"http://realt.by/agencies/.{5,100}</a>");
	Pattern agentBlock = Pattern.compile("<strong>.{5,70}<a href=\"http://realt.by/agencies/.{5,100}</a>");
	Pattern phoneNumber = Pattern.compile("(\\(.{5}\\)){1} ...-..-..");
	String tempUrl;
	//tempUrl = "http://realt.by/sale/flats/";
	tempUrl = request.getParameter("URL");
	URL u = new URL(tempUrl);
	String s;
	s = ReaderFactory.readUrlData(u,"ISO-8859-1");
	out.println(s);
	System.out.println(s);
	Matcher agentsMatcher = agentBlock.matcher(s);
	int count = 0;
    //OPen connection to datastore.
	Date date = new Date();
	String url = "realt.by";
	Agent newAgent;
	PersistenceManager pm = PMF.get().getPersistenceManager();
	
	while (agentsMatcher.find()) {
		count++;
		System.out.println("!!"+count);
		String content = agentsMatcher.group();
		System.out.println(content);
		String contentData = content.replaceAll("[a-z,A-Z,0-9,<,>,/,\\+,=,\",\\(,\\),-, ,:,\\.,\\?,_,-]*","");
		Matcher phoneMatcher = phoneNumber.matcher(content);
		System.out.println(contentData);
		while(phoneMatcher.find()){
		try{
			String phone = phoneMatcher.group();	
			System.out.println("phone"+phone);		
			phone = phone.replaceAll("[ ,(,),-]*","");
			phone = phone.replaceFirst("^\\+375|^80","");
			System.out.println("phone"+phone);
			Long phoneNumberParsed = Long.parseLong(phone);
		
		
			//Stage2. Writing two phones to datastore.
		
			newAgent =  new Agent(contentData, phoneNumberParsed,"cp1251", date, url);
			out.println("newAgent  "+newAgent.getPhoneNumber()+"        "+newAgent.getAgentName());
			pm.makePersistent(newAgent);
			

		}catch(Exception e)
		{out.print(e);	
			
		}	
		
		}
		out.println("!!"+count);
	}
	
	pm.close();
	
%>